{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "本小节对学习率进行调整"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"A very simple MNIST classifier.\n",
    "See extensive documentation at\n",
    "https://www.tensorflow.org/get_started/mnist/beginners\n",
    "\"\"\"\n",
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import argparse\n",
    "import sys\n",
    "\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting ./MNIST/train-images-idx3-ubyte.gz\n",
      "Extracting ./MNIST/train-labels-idx1-ubyte.gz\n",
      "Extracting ./MNIST/t10k-images-idx3-ubyte.gz\n",
      "Extracting ./MNIST/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "# 导入数据\n",
    "data_dir = './MNIST'\n",
    "mnist = input_data.read_data_sets(data_dir, one_hot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#定义数据\n",
    "x = tf.placeholder(tf.float32, [None, 784])   # 输入图片的大小，28x28=784\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])   # 输出0-9共10个数字\n",
    "learning_rate = tf.placeholder(tf.float32)    # 用于接收dropout操作的值，dropout为了防止过拟合\n",
    "\n",
    "with tf.name_scope('reshape'):\n",
    "#-1代表先不考虑输入的图片例子多少这个维度，后面的1是channel的数量，因为我们输入的图片是黑白的，因此channel是1，例如如果是RGB图像，那么channel就是3\n",
    "  x_image = tf.reshape(x, [-1, 28, 28, 1])    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#from keras.layers.initializers import he_normal\n",
    "# 卷积层定义\n",
    "#函数参数中的filter_size是指卷积核的大小,step表示布长\n",
    "#这里使用函数tf.contrib.layers.variance_scaling_initializer来对权重参数进行He/MRSA初始化，更改参数可以实现Xavier初始化\n",
    "def conv_op(input_op, filter_size, channel_out, name):\n",
    "    h_conv1 = tf.layers.conv2d(input_op, channel_out, [filter_size,filter_size],\n",
    "                             padding='SAME',\n",
    "                             activation=tf.nn.relu,name=name,kernel_initializer=tf.contrib.layers.variance_scaling_initializer())    \n",
    "    return h_conv1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 最大池化层\n",
    "def maxPool_op(input_op, filter_size, step, name):\n",
    "    h_pool1 = tf.layers.max_pooling2d(input_op, pool_size=[filter_size,filter_size],\n",
    "                        strides=[step, step], padding='VALID',name=name)\n",
    "    return h_pool1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\ndef full_connection(input_op, channel_out, name):\\n    channel_in = input_op.get_shape()[-1].value\\n    with tf.name_scope(name) as scope:\\n        weight = tf.Variable(tf.truncated_normal([channel_in, channel_out],mean=0,\\n                                                  dtype=tf.float32, stddev=0.1),\\n                                                  collections=[tf.GraphKeys.GLOBAL_VARIABLES,'WEIGHTS'])\\n        #weight = tf.get_variable(shape=[channel_in, channel_out], dtype=tf.float32,\\n        #                         initializer=xavier_initializer_conv2d(), name=scope + 'weight')\\n        bias = tf.Variable(tf.constant(value=0.0, shape=[channel_out], dtype=tf.float32), name='bias')\\n        input_op_reshape = tf.reshape(input_op, [-1, 7 * 7 * 64])\\n        fc = tf.nn.relu(tf.matmul(input_op_reshape, weight) + bias)\\n        return fc\\n\""
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 全连接层\n",
    "'''\n",
    "def full_connection(input_op, channel_out, name):\n",
    "    channel_in = input_op.get_shape()[-1].value\n",
    "    with tf.name_scope(name) as scope:\n",
    "        weight = tf.Variable(tf.truncated_normal([channel_in, channel_out],mean=0,\n",
    "                                                  dtype=tf.float32, stddev=0.1),\n",
    "                                                  collections=[tf.GraphKeys.GLOBAL_VARIABLES,'WEIGHTS'])\n",
    "        #weight = tf.get_variable(shape=[channel_in, channel_out], dtype=tf.float32,\n",
    "        #                         initializer=xavier_initializer_conv2d(), name=scope + 'weight')\n",
    "        bias = tf.Variable(tf.constant(value=0.0, shape=[channel_out], dtype=tf.float32), name='bias')\n",
    "        input_op_reshape = tf.reshape(input_op, [-1, 7 * 7 * 64])\n",
    "        fc = tf.nn.relu(tf.matmul(input_op_reshape, weight) + bias)\n",
    "        return fc\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#第一层卷积层，卷积核为5*5，深度为32，步长为1，输出为28*28*32\n",
    "conv1=conv_op(x_image,filter_size=5,channel_out=32,name='conv1')\n",
    "#第一个池化层，输出14*14*28\n",
    "pool1=maxPool_op(conv1,filter_size=2,step=2,name='pool1')\n",
    "#第二层卷积层，卷积核为5*5，深度为64，步长为1，输出为28*28*64\n",
    "conv2=conv_op(pool1,filter_size=5,channel_out=64,name='conv2')\n",
    "#第二个池化层，输出7*7*64\n",
    "pool2=maxPool_op(conv2,filter_size=2,step=2,name='pool2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.contrib.layers import flatten\n",
    "#全连接层，映射7*7*64特征图，映射为1024个特征\n",
    "with tf.name_scope('fc1'):\n",
    "  h_pool2_flat = flatten(pool2)\n",
    "  h_fc1 = tf.layers.dense(h_pool2_flat, 1024, activation=tf.nn.relu)\n",
    "\n",
    "# Dropout - controls the complexity of the model, prevents co-adaptation of\n",
    "# features.\n",
    "with tf.name_scope('dropout'):\n",
    "  keep_prob = tf.placeholder(tf.float32)\n",
    "  h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)\n",
    "\n",
    "# Map the 1024 features to 10 classes, one for each digit\n",
    "#这里同上，需要注意的是，最后暂不需要使用激活函数\n",
    "with tf.name_scope('fc2'):\n",
    "  y = tf.layers.dense(h_fc1_drop, 10, activation=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 设置正则化方法\n",
    "REGULARIZATION_RATE = 0.0001 # 比较合适的参数\n",
    "#REGULARIZATION_RATE = 0.001 # 比较合适的参数\n",
    "regularizer = tf.contrib.layers.l2_regularizer(REGULARIZATION_RATE)  # 定义L2正则化损失函数\n",
    "#regularization = regularizer(weights1) + regularizer(weights2)  # 计算模型的正则化损失"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 50, entropy loss: 0.411518, l2_loss: 0.088327, total loss: 0.499845\n",
      "0.96\n",
      "step 100, entropy loss: 0.294325, l2_loss: 0.088509, total loss: 0.382834\n",
      "0.95\n",
      "step 150, entropy loss: 0.251918, l2_loss: 0.088597, total loss: 0.340515\n",
      "0.95\n",
      "step 200, entropy loss: 0.238096, l2_loss: 0.088634, total loss: 0.326730\n",
      "0.97\n",
      "step 250, entropy loss: 0.073019, l2_loss: 0.088664, total loss: 0.161683\n",
      "0.99\n",
      "step 300, entropy loss: 0.172673, l2_loss: 0.088670, total loss: 0.261343\n",
      "0.98\n",
      "step 350, entropy loss: 0.161747, l2_loss: 0.088676, total loss: 0.250423\n",
      "0.97\n",
      "step 400, entropy loss: 0.121050, l2_loss: 0.088679, total loss: 0.209729\n",
      "0.98\n",
      "step 450, entropy loss: 0.128284, l2_loss: 0.088665, total loss: 0.216949\n",
      "0.99\n",
      "step 500, entropy loss: 0.096038, l2_loss: 0.088646, total loss: 0.184684\n",
      "0.97\n",
      "0.975\n",
      "step 550, entropy loss: 0.037793, l2_loss: 0.088620, total loss: 0.126413\n",
      "0.99\n",
      "step 600, entropy loss: 0.040891, l2_loss: 0.088605, total loss: 0.129496\n",
      "0.99\n",
      "step 650, entropy loss: 0.157370, l2_loss: 0.088581, total loss: 0.245951\n",
      "0.98\n",
      "step 700, entropy loss: 0.086023, l2_loss: 0.088560, total loss: 0.174583\n",
      "0.99\n",
      "step 750, entropy loss: 0.031870, l2_loss: 0.088536, total loss: 0.120406\n",
      "0.98\n",
      "step 800, entropy loss: 0.016946, l2_loss: 0.088520, total loss: 0.105466\n",
      "1.0\n",
      "step 850, entropy loss: 0.060460, l2_loss: 0.088484, total loss: 0.148944\n",
      "0.99\n",
      "step 900, entropy loss: 0.062964, l2_loss: 0.088452, total loss: 0.151416\n",
      "1.0\n",
      "step 950, entropy loss: 0.013348, l2_loss: 0.088421, total loss: 0.101769\n",
      "1.0\n",
      "step 1000, entropy loss: 0.030093, l2_loss: 0.088370, total loss: 0.118463\n",
      "0.99\n",
      "0.9816\n",
      "step 1050, entropy loss: 0.059851, l2_loss: 0.088325, total loss: 0.148176\n",
      "0.98\n",
      "step 1100, entropy loss: 0.099986, l2_loss: 0.088286, total loss: 0.188273\n",
      "0.97\n",
      "step 1150, entropy loss: 0.092755, l2_loss: 0.088246, total loss: 0.181001\n",
      "0.98\n",
      "step 1200, entropy loss: 0.006912, l2_loss: 0.088209, total loss: 0.095121\n",
      "1.0\n",
      "step 1250, entropy loss: 0.083969, l2_loss: 0.088163, total loss: 0.172132\n",
      "0.99\n",
      "step 1300, entropy loss: 0.018974, l2_loss: 0.088127, total loss: 0.107101\n",
      "0.99\n",
      "step 1350, entropy loss: 0.021872, l2_loss: 0.088083, total loss: 0.109955\n",
      "1.0\n",
      "step 1400, entropy loss: 0.021383, l2_loss: 0.088038, total loss: 0.109421\n",
      "1.0\n",
      "step 1450, entropy loss: 0.037114, l2_loss: 0.087990, total loss: 0.125104\n",
      "1.0\n",
      "step 1500, entropy loss: 0.121412, l2_loss: 0.087947, total loss: 0.209359\n",
      "0.98\n",
      "0.9858\n",
      "step 1550, entropy loss: 0.089334, l2_loss: 0.087898, total loss: 0.177232\n",
      "0.99\n",
      "step 1600, entropy loss: 0.033820, l2_loss: 0.087848, total loss: 0.121668\n",
      "1.0\n",
      "step 1650, entropy loss: 0.040083, l2_loss: 0.087796, total loss: 0.127880\n",
      "0.99\n",
      "step 1700, entropy loss: 0.016789, l2_loss: 0.087753, total loss: 0.104541\n",
      "1.0\n",
      "step 1750, entropy loss: 0.032614, l2_loss: 0.087706, total loss: 0.120321\n",
      "1.0\n",
      "step 1800, entropy loss: 0.015501, l2_loss: 0.087659, total loss: 0.103161\n",
      "1.0\n",
      "step 1850, entropy loss: 0.020245, l2_loss: 0.087611, total loss: 0.107856\n",
      "1.0\n",
      "step 1900, entropy loss: 0.012466, l2_loss: 0.087556, total loss: 0.100022\n",
      "0.99\n",
      "step 1950, entropy loss: 0.064098, l2_loss: 0.087509, total loss: 0.151608\n",
      "1.0\n",
      "step 2000, entropy loss: 0.020517, l2_loss: 0.087457, total loss: 0.107974\n",
      "1.0\n",
      "0.9871\n"
     ]
    }
   ],
   "source": [
    "cross_entropy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "regularization=0.0\n",
    "for w in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES):\n",
    "    regularization=regularization+regularizer(w)\n",
    "l2_loss=regularization\n",
    "#l2_loss = tf.add_n( [tf.nn.l2_loss(w) for w in tf.get_collection('WEIGHTS')] )\n",
    "#total_loss = cross_entropy + 7e-5*l2_loss\n",
    "total_loss = cross_entropy + l2_loss\n",
    "train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(total_loss)\n",
    "\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "# Train\n",
    "for step in range(2000):\n",
    "  batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "  #lr = 0.01\n",
    "  #lr = 0.001\n",
    "  lr = 0.1\n",
    "    \n",
    "    \n",
    "\n",
    "  _, loss, l2_loss_value, total_loss_value = sess.run(\n",
    "               [train_step, cross_entropy, l2_loss, total_loss], \n",
    "               feed_dict={x: batch_xs, y_: batch_ys, learning_rate:lr, keep_prob:0.5})\n",
    "  \n",
    "  if (step+1) % 50 == 0:\n",
    "    print('step %d, entropy loss: %f, l2_loss: %f, total loss: %f' % \n",
    "            (step+1, loss, l2_loss_value, total_loss_value))\n",
    "    # Test trained model\n",
    "    correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "    print(sess.run(accuracy, feed_dict={x: batch_xs, y_: batch_ys, keep_prob:0.5}))\n",
    "  if (step+1) % 500 == 0:\n",
    "    print(sess.run(accuracy, feed_dict={x: mnist.test.images,\n",
    "                                    y_: mnist.test.labels, keep_prob:0.5}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "心得与小结:\n",
    "这里可以看到学习率设置为0.1时效果更好。可以继续往大了设置。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
